<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Logging and Monitoring</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="Operations" href="index.html" />
    <link rel="next" title="Metrics" href="metrics.html" />
    <link rel="prev" title="Operations" href="index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: admin-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cdap-components.html"> CDAP Components</a></li>
<li class="toctree-l1"><a class="reference internal" href="../deployment-architectures.html"> Deployment Architectures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../hadoop-compatibility.html"> Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cdap-hadoop-compatibility.html"> CDAP and Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../system-requirements.html"> System Requirements</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation/index.html"> Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../installation/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/emr.html">Amazon EMR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/azure-hdinsight.html">Microsoft Azure HDInsight</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/packages.html">Packages</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/replication.html">Replication</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../incompatibilities.html"> Incompatibilities</a></li>
<li class="toctree-l1"><a class="reference internal" href="../upgrading/index.html"> Upgrading</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/packages.html">Packages</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> Security</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/perimeter-security.html">Perimeter Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization.html">Authorization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/impersonation.html">Impersonation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/system-services.html">Enabling SSL for System Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/secure-storage.html">Secure Storage</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> Operations</a><ul class="current">
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Logging and Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="metrics.html"> Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations-dashboard.html"> Dashboard and Reports</a></li>
<li class="toctree-l2"><a class="reference internal" href="preferences.html"> Preferences and Runtime Arguments</a></li>
<li class="toctree-l2"><a class="reference internal" href="scaling-instances.html"> Scaling Instances</a></li>
<li class="toctree-l2"><a class="reference internal" href="resource-guarantees.html"> Resource Guarantees in YARN</a></li>
<li class="toctree-l2"><a class="reference internal" href="tx-maintenance.html"> Transaction Service Maintenance</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-ui.html"> CDAP UI</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../appendices/index.html"> Appendices</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../appendices/cdap-site.html"> Appendix: cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/cdap-security.html"> Appendix: cdap-security.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/minimal-cdap-site.html"> Appendix: Minimal cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/hbase-ddl-executor.html"> Appendix: HBaseDDLExecutor</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="logging-and-monitoring">
<span id="logging-monitoring"></span><h1>Logging and Monitoring<a class="headerlink" href="#logging-and-monitoring" title="Permalink to this headline">🔗</a></h1>
<p>CDAP collects logs and metrics for all of its internal services and user applications.
Being able to view these details can be very helpful in debugging CDAP applications as
well as analyzing their performance. CDAP gives access to its logs, metrics, and other
monitoring information through RESTful APIs, the CDAP UI, as well as a Java Client.</p>
<p>In Hadoop clusters, the programs running inside their containers generate individual log files as
part of the container.
As an application can consist of
multiple  programs distributed across the nodes of a cluster, the complete logs for the
application may similarly be distributed.
As these files generally do not persist beyond the life of the
container, they are volatile and not very useful for post-mortem diagnostics and
troubleshooting, or for analyzing performance.</p>
<p>To address these issues, the CDAP log framework was designed:</p>
<ul class="simple">
<li>to centralize the location of logs, so that the logs of the individual containers of a
program can be merged into one;</li>
<li>to make logs both persistent—available for later use and analysis—and
available while the program is still running;</li>
<li>to be extensible using <a class="reference internal" href="#logging-monitoring-configuring-custom-log-pipelines"><span class="std std-ref">custom log pipelines</span></a>; and</li>
<li>to allow fine-tuning of the logging behavior at the level of an individual application
as well as the entire cluster.</li>
</ul>
<div class="section" id="logging-example">
<h2>Logging Example<a class="headerlink" href="#logging-example" title="Permalink to this headline">🔗</a></h2>
<p>This diagram shows the steps CDAP follows when logging a program of an application:</p>
<blockquote>
<div><ul class="simple">
<li>Logs are collected from an individual program running in a YARN container.</li>
<li>YARN writes the log messages emitted by containers to files inside the container.</li>
<li>In addition, CDAP programs publish these messages to Kafka.</li>
<li>The CDAP Log Saver Service is configured to read log messages from Kafka.
Log saver reads the messages from Kafka, groups them by program or application,
buffers and sorts them in memory, and finally persists them to files in HDFS. Each of
these files corresponds to one program or application, depending on how the grouping
is configured. (This is set by the property <a class="reference internal" href="#logging-monitoring-writing-logs-to-kafka"><span class="std std-ref">log.publish.partition.key</span></a>, described <a class="reference internal" href="#logging-monitoring-configuring-the-log-saver-service"><span class="std std-ref">below</span></a>.)</li>
<li>In addition to persisting logs to files, the Log.saver also emits metrics about the
number of log messages emitted by each program. These metrics can be retrieved by
querying the <span class="xref std std-ref">CDAP metrics system</span>.</li>
<li>For security, the files written out to persistent storage in HDFS have permissions set
such that they are accessible only by the <code class="docutils literal notranslate"><span class="pre">cdap</span></code> user.</li>
</ul>
</div></blockquote>
<div class="figure align-center" id="id2" style="width: 100%">
<a class="reference internal image-reference" href="../_images/logging-framework-simple.png"><img alt="../_images/logging-framework-simple.png" src="../_images/logging-framework-simple.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Logging Example:</strong> From a YARN container, through Kafka and the CDAP Log Saver Service, to HDFS</span></p>
</div>
<p>Logging uses the standard <a class="reference external" href="http://www.slf4j.org/manual.html">SLF4J (Simple Logging Facade for Java)</a> APIs and <a class="reference external" href="https://logback.qos.ch/manual">Logback</a>. Logging is configured using instances of Logback’s
“logback” file, consisting of <em>log pipelines</em> with <em>log appenders</em>:</p>
<blockquote>
<div><ul class="simple">
<li>A <strong>log pipeline</strong> is a process that consumes log events from Kafka, buffers, groups
by application or program, sorts, and then invokes the <em>log appenders</em> defined in its
configuration.</li>
<li>A <strong>log appender</strong> (or <em>appender</em>) is a Java class, responsible for consuming and
processing messages; typically, this includes persisting the log events. It can also,
for example, collect metrics, maintain metadata about the storage, or emit alerts when
it finds certain messages.</li>
</ul>
</div></blockquote>
</div>
<div class="section" id="user-application-program-logs">
<h2>User Application Program Logs<a class="headerlink" href="#user-application-program-logs" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="emitting-log-messages-from-a-program">
<h3>Emitting Log Messages from a Program<a class="headerlink" href="#emitting-log-messages-from-a-program" title="Permalink to this headline">🔗</a></h3>
<p>CDAP supports application logging through the standard <a class="reference external" href="http://www.slf4j.org/manual.html">SLF4J (Simple Logging Facade for Java)</a> APIs.</p>
<p>For instance, in a flowlet you can write:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">private</span> <span class="kd">static</span> <span class="n">Logger</span> <span class="n">LOG</span> <span class="o">=</span> <span class="n">LoggerFactory</span><span class="p">.</span><span class="na">getLogger</span><span class="p">(</span><span class="n">WordCounter</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
<span class="p">...</span>
<span class="nd">@ProcessInput</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">String</span> <span class="n">line</span><span class="p">)</span> <span class="p">{</span>
  <span class="n">LOG</span><span class="p">.</span><span class="na">debug</span><span class="p">(</span><span class="s">&quot;{}: Received line {}&quot;</span><span class="p">,</span> <span class="k">this</span><span class="p">.</span><span class="na">getContext</span><span class="p">().</span><span class="na">getTransactionAwareName</span><span class="p">(),</span> <span class="n">line</span><span class="p">);</span>
  <span class="p">...</span> <span class="c1">// processing</span>
  <span class="n">LOG</span><span class="p">.</span><span class="na">debug</span><span class="p">(</span><span class="s">&quot;{}: Emitting count {}&quot;</span><span class="p">,</span> <span class="k">this</span><span class="p">.</span><span class="na">getContext</span><span class="p">().</span><span class="na">getTransactionAwareName</span><span class="p">(),</span> <span class="n">wordCount</span><span class="p">);</span>
  <span class="n">output</span><span class="p">.</span><span class="na">emit</span><span class="p">(</span><span class="n">wordCount</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>This will emit “debug” level messages from the flowlet when it is processing an event.</p>
</div>
<div class="section" id="retrieving-log-messages-from-a-program">
<h3>Retrieving Log Messages from a Program<a class="headerlink" href="#retrieving-log-messages-from-a-program" title="Permalink to this headline">🔗</a></h3>
<p>The log messages emitted by your application code can be retrieved by:</p>
<ul>
<li><p class="first">Using the <span class="xref std std-ref">restful-api</span>: the <span class="xref std std-ref">Logging HTTP RESTful API</span> details the available contexts that can be called to
retrieve different messages.</p>
</li>
<li><p class="first">Log messages of a program can be viewed in the <a class="reference internal" href="cdap-ui.html#cdap-ui"><span class="std std-ref">CDAP UI</span></a>. In the
<em>Overview</em> page, select the application and program that you are interested in, and click
its <em>Logs</em> icon (circled in red here):</p>
<div class="figure align-center" id="id3" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/logging-cdap-ui-purchase.png"><img alt="../_images/logging-cdap-ui-purchase.png" class="bordered-image" src="../_images/logging-cdap-ui-purchase.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP UI:</strong> <em>Log</em> icon, enabled if there are logs available for viewing</span></p>
</div>
<p>The logs will be displayed in the log viewer:</p>
<div class="figure align-center" id="id4" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/logging-cdap-ui-purchase-flow-log.png"><img alt="../_images/logging-cdap-ui-purchase-flow-log.png" class="bordered-image" src="../_images/logging-cdap-ui-purchase-flow-log.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP UI:</strong> <em>Log Viewer</em>, showing <em>PurchaseFlow</em> log events, <em>INFO</em> level</span></p>
</div>
</li>
</ul>
</div>
<div class="section" id="program-log-file-locations">
<h3>Program Log File Locations<a class="headerlink" href="#program-log-file-locations" title="Permalink to this headline">🔗</a></h3>
<p>Program logs are stored in locations specified by properties in the
<a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">cdap-site.xml</span></a> file depending on the mode
of CDAP (Sandbox or Distributed):</p>
<ul class="simple">
<li>For <strong>CDAP Sandbox:</strong> the property <code class="docutils literal notranslate"><span class="pre">log.collection.root</span></code> (default
<code class="docutils literal notranslate"><span class="pre">${local.data.dir}/logs</span></code>) is the root location for collecting program logs when in
CDAP Sandbox.</li>
<li>For <strong>Distributed CDAP:</strong> the property <code class="docutils literal notranslate"><span class="pre">hdfs.namespace</span></code> (default <code class="docutils literal notranslate"><span class="pre">/cdap</span></code>) is the
base directory in HDFS; program logs are stored in <code class="docutils literal notranslate"><span class="pre">${hdfs.namespace}/logs</span></code> (by default,
<code class="docutils literal notranslate"><span class="pre">/cdap/logs</span></code>).</li>
</ul>
</div>
<div class="section" id="configuring-program-logs-and-log-levels">
<span id="configuring-user-application-program-logging"></span><h3>Configuring Program Logs and Log Levels<a class="headerlink" href="#configuring-program-logs-and-log-levels" title="Permalink to this headline">🔗</a></h3>
<p>The logging of an application’s programs are configured by the <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code>
file, packaged with the CDAP distribution. This “logback” does log rotation once a day at
midnight and expires logs older than 14 days. Changes can be made to
<code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code>; afterwards, applications or programs needs to be restarted
for the modified logback file to take effect. Changing the <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> will
only affect programs that are started after the change; existing running programs will not
be affected.</p>
<ul class="simple">
<li>For <strong>CDAP Sandbox:</strong> As the entire CDAP Sandbox runs in a single JVM, the
<code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file, located in  <code class="docutils literal notranslate"><span class="pre">&lt;CDAP-HOME&gt;/conf</span></code>, configures both “container”
and CDAP system service logging.</li>
<li>For <strong>Distributed CDAP:</strong> the <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> file is located in <code class="docutils literal notranslate"><span class="pre">/etc/cdap/conf</span></code>.</li>
</ul>
<p>You can also use a custom “logback” file with your application, as described in the
开发手册 section <a class="reference external" href="../../../developer-manual/advanced/application-logback.html#application-logback" title="(in Cask Data Application Platform v6.1.1)"><span>Application Logback</span></a>.</p>
</div>
<div class="section" id="changing-program-log-levels">
<h3>Changing Program Log Levels<a class="headerlink" href="#changing-program-log-levels" title="Permalink to this headline">🔗</a></h3>
<p>When running under Distributed CDAP, the log levels of a program can be changed without
modifying the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> or <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> files. This can be done, for
all program types, before a particular run or—in the case of a flow, service, or
worker—while it is running.</p>
<p>The CDAP <span class="xref std std-ref">Logging HTTP RESTful API</span> can be used to set the log levels
of a flow, service, or worker which is running. Once changed, they can be reset back to what they
were originally by using the <span class="xref std std-ref">reset endpoint</span>.</p>
<p>Only flows, services, or workers can be dynamically changed; other program types are
currently not supported. The other program types’ log levels can only be changed using
their preferences before the program starts.</p>
<ul>
<li><p class="first">To configure the log level before an application starts, you can add the logger name as the
key and log level as the value in the <span class="xref std std-ref">preferences</span>,
using the <a class="reference internal" href="cdap-ui.html#cdap-ui"><span class="std std-ref">CDAP UI</span></a>, <span class="xref std std-ref">CDAP CLI</span>, or other command line
tools. The logger name should be prefixed with <code class="docutils literal notranslate"><span class="pre">system.log.level</span></code>.</p>
<p>For example, if you want to set the log level of a class named <code class="docutils literal notranslate"><span class="pre">MyService</span></code> with package
<code class="docutils literal notranslate"><span class="pre">my.sample.package</span></code> to <code class="docutils literal notranslate"><span class="pre">DEBUG</span></code>, you would use
<code class="docutils literal notranslate"><span class="pre">system.log.level.my.sample.package.MyService</span></code> as the key and <code class="docutils literal notranslate"><span class="pre">DEBUG</span></code>
as the value. This can be applied to any package or classes. If the logger name is
omitted, it will change the log level of ROOT.</p>
</li>
<li><p class="first">To configure the log level of a program dynamically—such as a flow, service, or worker
which is currently running—see the <span class="xref std std-ref">Logging HTTP RESTful API</span>.</p>
</li>
</ul>
<p><strong>Note:</strong> The <span class="xref std std-ref">Logging HTTP RESTful API</span> for changing program log levels
can only be used with programs that are running under Distributed CDAP. For changing the log
levels of programs run under CDAP Sandbox, you either modify the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file,
or you <a class="reference external" href="../../../developer-manual/advanced/application-logback.html#application-logback" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">provide a “logback.xml”</span></a> with your application
before it is deployed.</p>
</div>
</div>
<div class="section" id="cdap-system-services-logs">
<span id="logging-monitoring-system-services-logs"></span><h2>CDAP System Services Logs<a class="headerlink" href="#cdap-system-services-logs" title="Permalink to this headline">🔗</a></h2>
<p>As CDAP system services run either on cluster edge nodes or in YARN containers, their
logging and <a class="reference internal" href="#logging-monitoring-system-services-logs-configuration"><span class="std std-ref">its configuration</span></a>
depends on the service and where it is located.</p>
<div class="section" id="retrieving-log-messages-from-a-system-service">
<h3>Retrieving Log Messages from a System Service<a class="headerlink" href="#retrieving-log-messages-from-a-system-service" title="Permalink to this headline">🔗</a></h3>
<p>The log messages emitted by CDAP system services can be retrieved by:</p>
<ul>
<li><p class="first">Using the <span class="xref std std-ref">restful-api</span>: the <span class="xref std std-ref">Logging HTTP RESTful API</span> details <span class="xref std std-ref">downloading the logs</span> emitted by a system service.</p>
</li>
<li><p class="first">Log messages of system services can be viewed in the <a class="reference internal" href="cdap-ui.html#cdap-ui"><span class="std std-ref">CDAP UI</span></a>. In the
<em>Administration</em> page (accessible through the <em>CDAP</em> menu on the far-right) click either
a <em>Logs</em> icon (such as the <em>HBase</em> log icon circled in red, in the <em>Component Overview</em>
section):</p>
<div class="figure align-center" id="id5" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/logging-cdap-ui-administration.png"><img alt="../_images/logging-cdap-ui-administration.png" class="bordered-image" src="../_images/logging-cdap-ui-administration.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP UI:</strong> HBase Component, with its <em>Log</em> icon circled in red</span></p>
</div>
<p>You can click on a CDAP system service and a window will appear displaying a log icon
(in this case, after clicking <em>App Fabric</em>), as shown here circled in red:</p>
<div class="figure align-center" id="id6" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/logging-cdap-ui-administration-app-fabric.png"><img alt="../_images/logging-cdap-ui-administration-app-fabric.png" class="bordered-image" src="../_images/logging-cdap-ui-administration-app-fabric.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP UI:</strong> CDAP <em>App Fabric</em> popup window, with its <em>Log</em> icon circled in red</span></p>
</div>
</li>
</ul>
</div>
<div class="section" id="system-service-log-file-locations">
<h3>System Service Log File Locations<a class="headerlink" href="#system-service-log-file-locations" title="Permalink to this headline">🔗</a></h3>
<p>The location of CDAP system service logs depends on the mode of CDAP (Sandbox or
Distributed) and the Hadoop distribution:</p>
<ul class="simple">
<li>For <strong>CDAP Sandbox:</strong> system logs are located in <code class="docutils literal notranslate"><span class="pre">&lt;CDAP-HOME&gt;/logs</span></code>.</li>
<li>For <strong>Distributed CDAP:</strong> system logs are located in <code class="docutils literal notranslate"><span class="pre">/var/log/cdap</span></code> (with the
exception of Cloudera Manager-based clusters). With Cloudera Manager installations, system
log files are located in directories under <code class="docutils literal notranslate"><span class="pre">/var/run/cloudera-scm-agent/process</span></code>.</li>
</ul>
</div>
<div class="section" id="configuring-system-service-logs">
<span id="logging-monitoring-system-services-logs-configuration"></span><h3>Configuring System Service Logs<a class="headerlink" href="#configuring-system-service-logs" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>CDAP system services that run in YARN containers, such as the <em>Metrics Service</em>, are
configured by the same <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> that <a class="reference internal" href="#configuring-user-application-program-logging"><span class="std std-ref">configures user application
program logging</span></a>.</li>
<li>CDAP system services that run on cluster edge nodes, such as CDAP Master or Router, are
configured by the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> Changes can be made to <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code>; afterwards,
the service(s) affected will need to be restarted for the modified “logback” file to
take effect.<ul>
<li>For <strong>CDAP Sandbox:</strong> the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file is located in <code class="docutils literal notranslate"><span class="pre">/etc/cdap/conf</span></code>.</li>
<li>For <strong>Distributed CDAP:</strong> the file <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file, located in
<code class="docutils literal notranslate"><span class="pre">&lt;CDAP-HOME&gt;/conf</span></code>, configures both “container” and CDAP system service logging.</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="changing-system-service-log-levels">
<h3>Changing System Service Log Levels<a class="headerlink" href="#changing-system-service-log-levels" title="Permalink to this headline">🔗</a></h3>
<p>When running under Distributed CDAP, the log levels of system services can be changed at
runtime without either modifying the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> or restarting CDAP.</p>
<p>The CDAP <span class="xref std std-ref">Logging HTTP RESTful API</span> can be used to set the log
levels of a system service while it is running. Once changed, they can be reset back to
what they were originally by using the <span class="xref std std-ref">reset endpoint</span>.</p>
<p>The REST endpoints can be applied to all system services listed at <span class="xref std std-ref">Logging HTTP
RESTful API</span>. However, since
<code class="docutils literal notranslate"><span class="pre">appfabric</span></code> and <code class="docutils literal notranslate"><span class="pre">dataset.service</span></code> are running on the same node, changing log levels of the
<code class="docutils literal notranslate"><span class="pre">appfabric</span></code> service <strong>will also</strong> change the log levels of the <code class="docutils literal notranslate"><span class="pre">dataset.service</span></code>.</p>
<p><strong>Note:</strong> The <span class="xref std std-ref">Logging HTTP RESTful API</span> for changing system service
log levels can only be used with system services that are running under Distributed CDAP.
For changing the log levels of system services under CDAP Sandbox, you need to modify
the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file and restart CDAP.</p>
</div>
</div>
<div class="section" id="configuring-the-log-saver-service">
<span id="logging-monitoring-configuring-the-log-saver-service"></span><h2>Configuring the Log Saver Service<a class="headerlink" href="#configuring-the-log-saver-service" title="Permalink to this headline">🔗</a></h2>
<p>The Log Saver Service is the CDAP service that reads log messages from Kafka, processes
them in <em>log pipelines</em>, persists them to HDFS, and sends metrics on logging to the
Metrics Service.</p>
<p>In addition to the default <em>CDAP Log Pipeline</em>, you can specify <a class="reference internal" href="#logging-monitoring-custom-log-pipelines"><span class="std std-ref">custom log pipelines</span></a> that are run by the log saver service and
perform custom actions.</p>
<p>The <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-site-xml"><span class="std std-ref">cdap-site.xml</span></a> file has properties that control the
writing of logs to Kafka, the log saver service, the CDAP log pipeline, and any
<a class="reference internal" href="#logging-monitoring-custom-log-pipelines"><span class="std std-ref">custom log pipelines</span></a> that have been defined.</p>
<div class="section" id="writing-logs-to-kafka">
<span id="logging-monitoring-writing-logs-to-kafka"></span><h3>Writing Logs to Kafka<a class="headerlink" href="#writing-logs-to-kafka" title="Permalink to this headline">🔗</a></h3>
<p>These <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">properties</span></a> control the writing of logs to
Kafka:</p>
<table border="1" class="docutils">
<colgroup>
<col width="30%" />
<col width="35%" />
<col width="35%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter Name</th>
<th class="head">Default Value</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.kafka.topic</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">logs.user-v2</span></code></td>
<td>Kafka topic name used to publish logs</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.publish.num.partitions</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">10</span></code></td>
<td>Number of CDAP Kafka service partitions to publish the logs to</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.publish.partition.key</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">program</span></code></td>
<td>Publish logs from an application or a program to the same partition. Valid values are “application” or “program”. If set to “application”, logs from all the programs of an application go to the same partition. If set to “program”, logs from the same program go to the same partition. Changes to this property requires restarting of all CDAP applications.</td>
</tr>
</tbody>
</table>
<p><strong>Notes:</strong></p>
<ul class="simple">
<li>If an external Kafka service is used (instead of the CDAP Kafka service), the
number of partitions used for <code class="docutils literal notranslate"><span class="pre">log.publish.num.partitions</span></code> must match the number set
in the external service for the topic being used to publish logs (<code class="docutils literal notranslate"><span class="pre">log.kafka.topic</span></code>).</li>
<li>By default, <code class="docutils literal notranslate"><span class="pre">log.publish.partition.key</span></code> is set to <code class="docutils literal notranslate"><span class="pre">program</span></code>, which means that all
logs for the same program go to the same partition. Set this to <code class="docutils literal notranslate"><span class="pre">application</span></code> if you
want all logs from an application to go to the same instance of the Log Saver Service.</li>
</ul>
</div>
<div class="section" id="log-saver-service">
<h3>Log Saver Service<a class="headerlink" href="#log-saver-service" title="Permalink to this headline">🔗</a></h3>
<p>These <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">properties</span></a> control the Log Saver Service:</p>
<table border="1" class="docutils">
<colgroup>
<col width="30%" />
<col width="35%" />
<col width="35%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter Name</th>
<th class="head">Default Value</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.saver.max.instances</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">${master.service.max.instances}</span></code></td>
<td>Maximum number of log saver instances to run in YARN</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.saver.num.instances</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">1</span></code></td>
<td>Number of log saver instances to run in YARN</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.saver.container.memory.mb</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">${master.service.memory.mb}</span></code></td>
<td>Memory in megabytes for each log saver instance to run in YARN.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.saver.container.num.cores</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">2</span></code></td>
<td>Number of virtual cores for each log saver instance in YARN</td>
</tr>
</tbody>
</table>
<p>Log saver instances should be from a minimum of one to a maximum of ten. The maximum is
set by the number of Kafka partitions (<code class="docutils literal notranslate"><span class="pre">log.publish.num.partitions</span></code>), which by default is 10.</p>
</div>
<div class="section" id="log-pipeline-configuration">
<span id="logging-monitoring-log-pipeline-configuration"></span><h3>Log Pipeline Configuration<a class="headerlink" href="#log-pipeline-configuration" title="Permalink to this headline">🔗</a></h3>
<p>Configuration properties for logging and <a class="reference internal" href="#logging-monitoring-configuring-custom-log-pipelines"><span class="std std-ref">custom log pipelines</span></a> are shown in the documentation of the
<a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">logging properties</span></a> section of the
<a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-site-xml"><span class="std std-ref">cdap-site.xml</span></a> file.</p>
<p>The <strong>CDAP log pipeline</strong> is configured by settings in the <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-site-xml"><span class="std std-ref">cdap-site.xml</span></a> file.</p>
<p><strong>Custom log pipelines</strong> are configured by a combination of the settings in the
<a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-site-xml"><span class="std std-ref">cdap-site.xml</span></a> file and a “logback” file used
to specify the custom pipeline. The XML file is placed in the
<code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code>, a local directory on the CDAP Master node that is scanned
for log processing pipeline configurations. Each pipeline is defined by a file in the
Logback XML format, with <code class="docutils literal notranslate"><span class="pre">.xml</span></code> as the file name extension.</p>
<p>These <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">properties</span></a> control the CDAP log pipeline:</p>
<table border="1" class="docutils">
<colgroup>
<col width="30%" />
<col width="35%" />
<col width="35%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter Name</th>
<th class="head">Default Value</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.dir.permissions</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">700</span></code></td>
<td>Permissions used by the system log pipeline when creating directories</td>
</tr>
<tr class="row-odd"><td><div class="first last line-block">
<div class="line"><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.cleanup.batch.</span></code></div>
<div class="line"><code class="docutils literal notranslate"><span class="pre">size</span></code></div>
</div>
</td>
<td><code class="docutils literal notranslate"><span class="pre">10000</span></code></td>
<td>Batch size to clean up log metadata table</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.lifetime.ms</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">21600000</span></code></td>
<td>Maximum time span in milliseconds of a log file created by the system log pipeline</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.size.bytes</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">104857600</span></code></td>
<td>Maximum size in bytes of a log file created by the system log pipeline</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.permissions</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">600</span></code></td>
<td>Permissions used by the system log pipeline when creating files</td>
</tr>
<tr class="row-odd"><td><div class="first last line-block">
<div class="line"><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.retention.</span></code></div>
<div class="line"><code class="docutils literal notranslate"><span class="pre">duration.days</span></code></div>
</div>
</td>
<td><code class="docutils literal notranslate"><span class="pre">7</span></code></td>
<td>Time in days a log file is retained</td>
</tr>
</tbody>
</table>
<p id="logging-monitoring-cdap-site-xml-configuration">These <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">properties</span></a> control both the CDAP log pipeline
and custom log pipelines:</p>
<table border="1" class="docutils">
<colgroup>
<col width="30%" />
<col width="35%" />
<col width="35%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter Name</th>
<th class="head">Default Value</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><div class="first last line-block">
<div class="line"><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.checkpoint.</span></code></div>
<div class="line"><code class="docutils literal notranslate"><span class="pre">interval.ms</span></code></div>
</div>
</td>
<td><code class="docutils literal notranslate"><span class="pre">10000</span></code></td>
<td>The time between log processing pipeline checkpoints in milliseconds</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">/opt/cdap/master/ext/logging/config</span></code></td>
<td>A local directory on the CDAP Master that is scanned for log processing pipeline configurations. Each pipeline is defined by a file in the logback XML format, with “.xml” as the file name extension.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.event.delay.ms</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">2000</span></code></td>
<td>The time a log event stays in the log processing pipeline buffer before writing out to log appenders in milliseconds. A longer delay will result in better time ordering of log events before presenting to log appenders but will consume more memory.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.kafka.fetch.size</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">1048576</span></code></td>
<td>The buffer size in bytes, per topic partition, for fetching log events from Kafka</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.lib.dir</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">/opt/cdap/master/ext/logging/lib</span></code></td>
<td>Comma-separated list of local directories on the CDAP Master scanned for additional library JAR files to be included for log processing</td>
</tr>
</tbody>
</table>
<p>The <code class="docutils literal notranslate"><span class="pre">log.process.pipeline.*</span></code> properties can be over-ridden and specified at the
custom pipeline level by providing a value in a pipeline’s “logback” file for any of
these properties.</p>
</div>
</div>
<div class="section" id="logging-framework">
<h2>Logging Framework<a class="headerlink" href="#logging-framework" title="Permalink to this headline">🔗</a></h2>
<p>This diagram shows in greater detail the components and steps CDAP follows when logging
programs of an application and system services with the logging framework:</p>
<div class="figure align-center" id="id7" style="width: 100%">
<a class="reference internal image-reference" href="../_images/logging-framework.png"><img alt="../_images/logging-framework.png" src="../_images/logging-framework.png" style="width: 800px;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Logging Framework:</strong> From YARN containers, through Kafka and the Log Saver
Service, to HDFS</span></p>
</div>
<ul>
<li><p class="first">Logs are collected from individual programs running in YARN containers.</p>
</li>
<li><p class="first">YARN writes the log messages emitted by containers to files inside the container.</p>
</li>
<li><p class="first">In addition, CDAP programs publish these messages to Kafka.</p>
</li>
<li><p class="first">CDAP System Services run (depending on the service) either on cluster edge nodes or in
YARN containers. Where they run determines the file that configures that service’s
logging.</p>
</li>
<li><p class="first">The Log Saver Service (<em>log.saver</em>) is configured to read log messages for the
<code class="docutils literal notranslate"><span class="pre">logs.user-v2</span></code> Kafka topic (set by the property <code class="docutils literal notranslate"><span class="pre">log.kafka.topic</span></code>). The number of
log saver instances can be scaled to process the Kafka partitions in parallel, if
needed.</p>
<p>Log saver, by default, runs only the CDAP Log Pipeline: it reads the messages from
Kafka, groups them by program or application, buffers and sorts them in memory, and
finally persists them to files in HDFS. Each of these files corresponds to one program
or application, depending on how the grouping is configured. (This is set by the
property <a class="reference internal" href="#logging-monitoring-writing-logs-to-kafka"><span class="std std-ref">log.publish.partition.key</span></a>,
described <a class="reference internal" href="#logging-monitoring-configuring-the-log-saver-service"><span class="std std-ref">below</span></a>.)</p>
<p><em>Note:</em> These files are configured to rotate based on time and size; they can be
changed using the properties <code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.size.bytes</span></code> and
<code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.lifetime.ms</span></code> in the <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">cdap-site.xml</span></a> file as described in
<a class="reference internal" href="#logging-monitoring-log-pipeline-configuration"><span class="std std-ref">Log Pipeline Configuration</span></a>.</p>
</li>
<li><p class="first">For security, the files written out to persistent storage in HDFS have permissions set
such that they are accessible only by the <code class="docutils literal notranslate"><span class="pre">cdap</span></code> user.</p>
</li>
<li><p class="first">In addition, custom log pipelines can be configured by adding an XML file in a
prescribed location. Each pipeline buffers log messages in memory and sorts them based
on their timestamp.</p>
</li>
<li><p class="first">In addition to persisting logs to files, the log saver also emits metrics about the
number of log messages emitted by each program. These metrics can be retrieved by
querying the <span class="xref std std-ref">CDAP metrics system</span>.</p>
<p>These tables list the metrics from the section <span class="xref std std-ref">available-system-metrics</span> of the
<span class="xref std std-ref">http-restful-api-metrics</span>. See that section for further information.</p>
<table border="1" class="docutils">
<colgroup>
<col width="50%" />
<col width="50%" />
</colgroup>
<tbody valign="top">
<tr class="row-odd"><td><strong>Application Logging Metric</strong></td>
<td><strong>Description</strong></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">system.app.log.{error,</span> <span class="pre">info,</span> <span class="pre">warn}</span></code></td>
<td>Number of <code class="docutils literal notranslate"><span class="pre">error</span></code>, <code class="docutils literal notranslate"><span class="pre">info</span></code>, or <code class="docutils literal notranslate"><span class="pre">warn</span></code> log messages logged by an application or
applications</td>
</tr>
<tr class="row-odd"><td><strong>System Services Logging Metric</strong></td>
<td><strong>Description</strong></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">system.services.log.{error,</span> <span class="pre">info,</span> <span class="pre">warn}</span></code></td>
<td>Number of <code class="docutils literal notranslate"><span class="pre">error</span></code>, <code class="docutils literal notranslate"><span class="pre">info</span></code>, or <code class="docutils literal notranslate"><span class="pre">warn</span></code> log messages logged by a system service
or system services</td>
</tr>
</tbody>
</table>
</li>
</ul>
</div>
<div class="section" id="custom-log-pipelines">
<span id="logging-monitoring-custom-log-pipelines"></span><h2>Custom Log Pipelines<a class="headerlink" href="#custom-log-pipelines" title="Permalink to this headline">🔗</a></h2>
<p>For a custom log pipeline, create and configure a “logback” file,
configuring loggers, appenders, and properties based on your requirements, and place the
file at the path specified in the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> file by the property
<code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code> of the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> file.</p>
<p>Each custom pipeline requires a unique name. Properties controlling the pipeline (the
<code class="docutils literal notranslate"><span class="pre">log.process.pipeline.*</span></code> properties) are described <a class="reference internal" href="#logging-monitoring-cdap-site-xml-configuration"><span class="std std-ref">above</span></a>.</p>
<p>For every XML file in the <code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code> directory, a separate log
pipeline is created. As they are separate Kafka consumers and processes, each pipeline is
isolated and independent of each other. The performance of one pipeline does not affect
the performance of another. Though CDAP has been tested with multiple log pipelines and
appenders, the fewer of each that are specified will provide better performance.</p>
<div class="section" id="configuring-custom-log-pipelines">
<span id="logging-monitoring-configuring-custom-log-pipelines"></span><h3>Configuring Custom Log Pipelines<a class="headerlink" href="#configuring-custom-log-pipelines" title="Permalink to this headline">🔗</a></h3>
<p>CDAP looks for “logback” files located in a directory as set by the property
<code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code> in the <a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-default-logging"><span class="std std-ref">cdap-site.xml</span></a> file. In the default configuration, this is:</p>
<ul class="simple">
<li>For <strong>CDAP Sandbox:</strong> <code class="docutils literal notranslate"><span class="pre">&lt;CDAP-HOME&gt;/ext/logging/config</span></code></li>
<li>For <strong>Distributed CDAP:</strong> <code class="docutils literal notranslate"><span class="pre">/opt/cdap/master/ext/logging/config</span></code></li>
</ul>
</div>
<div class="section" id="example-logback-file-for-a-custom-log-pipeline">
<span id="logging-monitoring-custom-logging-example"></span><h3>Example “logback” File for a Custom log pipeline<a class="headerlink" href="#example-logback-file-for-a-custom-log-pipeline" title="Permalink to this headline">🔗</a></h3>
<p>Here is an example “logback” file, using two appenders (<code class="docutils literal notranslate"><span class="pre">STDOUT</span></code> and
<code class="docutils literal notranslate"><span class="pre">rollingAppender</span></code>). This file must be located (as <a class="reference internal" href="#logging-monitoring-configuring-custom-log-pipelines"><span class="std std-ref">noted above</span></a>) with a file extension of <code class="docutils literal notranslate"><span class="pre">.xml</span></code>:</p>
<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span><span class="cp">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span>
<span class="nt">&lt;configuration&gt;</span>
  <span class="nt">&lt;appender</span> <span class="na">name=</span><span class="s">&quot;STDOUT&quot;</span> <span class="na">class=</span><span class="s">&quot;ch.qos.logback.core.ConsoleAppender&quot;</span><span class="nt">&gt;</span>
    <span class="nt">&lt;encoder&gt;</span>
      <span class="nt">&lt;pattern&gt;</span>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n<span class="nt">&lt;/pattern&gt;</span>
    <span class="nt">&lt;/encoder&gt;</span>
  <span class="nt">&lt;/appender&gt;</span>

  <span class="nt">&lt;property</span> <span class="na">name=</span><span class="s">&quot;cdap.log.saver.instance.id&quot;</span> <span class="na">value=</span><span class="s">&quot;instanceId&quot;</span><span class="nt">/&gt;</span>

  <span class="nt">&lt;appender</span> <span class="na">name=</span><span class="s">&quot;rollingAppender&quot;</span> <span class="na">class=</span><span class="s">&quot;io.cdap.cdap.logging.plugins.RollingLocationLogAppender&quot;</span><span class="nt">&gt;</span>

    <span class="c">&lt;!-- log file path will be created by the appender as: &lt;basePath&gt;/&lt;namespace-id&gt;/&lt;application-id&gt;/&lt;filePath&gt; --&gt;</span>
    <span class="nt">&lt;basePath&gt;</span>plugins/applogs<span class="nt">&lt;/basePath&gt;</span>
    <span class="nt">&lt;filePath&gt;</span>securityLogs/logFile-${cdap.log.saver.instance.id}.log<span class="nt">&lt;/filePath&gt;</span>

    <span class="c">&lt;!-- cdap is the owner of the log files directory, so cdap will get read/write/execute permissions.</span>
<span class="c">    Log files will be read-only for others. --&gt;</span>
    <span class="nt">&lt;dirPermissions&gt;</span>744<span class="nt">&lt;/dirPermissions&gt;</span>

    <span class="c">&lt;!-- cdap is the owner of the log files, so cdap will get read/write permissions.</span>
<span class="c">    Log files will be read-only for others --&gt;</span>
    <span class="nt">&lt;filePermissions&gt;</span>644<span class="nt">&lt;/filePermissions&gt;</span>

    <span class="c">&lt;!-- It is an optional parameter, which takes number of miliseconds.</span>
<span class="c">    Appender will close a file if it is not modified for fileMaxInactiveTimeMs</span>
<span class="c">    period of time. Here it is set for thirty minutes. --&gt;</span>
    <span class="nt">&lt;fileMaxInactiveTimeMs&gt;</span>1800000<span class="nt">&lt;/fileMaxInactiveTimeMs&gt;</span>

    <span class="nt">&lt;rollingPolicy</span> <span class="na">class=</span><span class="s">&quot;io.cdap.cdap.logging.plugins.FixedWindowRollingPolicy&quot;</span><span class="nt">&gt;</span>
      <span class="c">&lt;!-- Only specify the file name without a directory, as the appender will use the</span>
<span class="c">      appropriate directory specified in filePath --&gt;</span>
      <span class="nt">&lt;fileNamePattern&gt;</span>logFile-${cdap.log.saver.instance.id}.log.%i<span class="nt">&lt;/fileNamePattern&gt;</span>
      <span class="nt">&lt;minIndex&gt;</span>1<span class="nt">&lt;/minIndex&gt;</span>
      <span class="nt">&lt;maxIndex&gt;</span>9<span class="nt">&lt;/maxIndex&gt;</span>
    <span class="nt">&lt;/rollingPolicy&gt;</span>

    <span class="nt">&lt;triggeringPolicy</span> <span class="na">class=</span><span class="s">&quot;io.cdap.cdap.logging.plugins.SizeBasedTriggeringPolicy&quot;</span><span class="nt">&gt;</span>
      <span class="c">&lt;!-- Set the maximum file size appropriately to avoid a large number of small files --&gt;</span>
      <span class="nt">&lt;maxFileSize&gt;</span>100MB<span class="nt">&lt;/maxFileSize&gt;</span>
    <span class="nt">&lt;/triggeringPolicy&gt;</span>

    <span class="nt">&lt;encoder&gt;</span>
      <span class="nt">&lt;pattern&gt;</span>%-4relative [%thread] %-5level %logger{35} - %msg%n<span class="nt">&lt;/pattern&gt;</span>
      <span class="c">&lt;!-- Do not flush on every event --&gt;</span>
      <span class="nt">&lt;immediateFlush&gt;</span>false<span class="nt">&lt;/immediateFlush&gt;</span>
    <span class="nt">&lt;/encoder&gt;</span>
  <span class="nt">&lt;/appender&gt;</span>

  <span class="nt">&lt;logger</span> <span class="na">name=</span><span class="s">&quot;io.cdap.cdap.logging.plugins.RollingLocationLogAppenderTest&quot;</span> <span class="na">level=</span><span class="s">&quot;INFO&quot;</span><span class="nt">&gt;</span>
    <span class="nt">&lt;appender-ref</span> <span class="na">ref=</span><span class="s">&quot;rollingAppender&quot;</span><span class="nt">/&gt;</span>
  <span class="nt">&lt;/logger&gt;</span>

  <span class="nt">&lt;root</span> <span class="na">level=</span><span class="s">&quot;INFO&quot;</span><span class="nt">&gt;</span>
    <span class="nt">&lt;appender-ref</span> <span class="na">ref=</span><span class="s">&quot;STDOUT&quot;</span><span class="nt">/&gt;</span>
  <span class="nt">&lt;/root&gt;</span>

<span class="nt">&lt;/configuration&gt;</span>
</pre></div>
</div>
</div>
<div class="section" id="custom-log-appender">
<h3>Custom Log Appender<a class="headerlink" href="#custom-log-appender" title="Permalink to this headline">🔗</a></h3>
<p>You can use any existing <a class="reference external" href="https://logback.qos.ch/manual/appenders.html">logback</a>
appender. The <code class="docutils literal notranslate"><span class="pre">RollingLocationLogAppender</span></code>—an extension of the Logback
<code class="docutils literal notranslate"><span class="pre">FileAppender</span></code>—lets you use HDFS locations in your log pipelines.</p>
<p>If you need an appender beyond what is available through Logback or CDAP, you can write
and implement your own custom appender. See the <a class="reference external" href="https://logback.qos.ch/manual/appenders.html">Logback documentation</a> for information on this.</p>
<p>As the CDAP LogFramework uses Logback’s Appender API, your custom appender needs to
implement the same Appender interface. Access to CDAP’s system components (such as
datasets, metrics, <code class="docutils literal notranslate"><span class="pre">LocationFactory</span></code>) are made available to the <code class="docutils literal notranslate"><span class="pre">AppenderContext</span></code>, an
extension of Logback’s <code class="docutils literal notranslate"><span class="pre">LoggerContext</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">CustomLogAppender</span> <span class="kd">extends</span> <span class="n">FileAppender</span><span class="o">&lt;</span><span class="n">ILoggingEvent</span><span class="o">&gt;</span> <span class="kd">implements</span> <span class="n">Flushable</span><span class="p">,</span> <span class="n">Syncable</span> <span class="p">{</span>
  <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
  <span class="kd">private</span> <span class="n">LocationManager</span> <span class="n">locationManager</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">start</span><span class="p">()</span> <span class="p">{</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">context</span> <span class="k">instanceof</span> <span class="n">AppenderContext</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">AppenderContext</span> <span class="n">context</span> <span class="o">=</span> <span class="p">(</span><span class="n">AppenderContext</span><span class="p">)</span> <span class="k">this</span><span class="p">.</span><span class="na">context</span><span class="p">;</span>
      <span class="n">locationManager</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocationManager</span><span class="p">(</span><span class="n">context</span><span class="p">.</span><span class="na">getLocationFactory</span><span class="p">()</span> <span class="p">.</span> <span class="p">.</span> <span class="p">.);</span>
      <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
    <span class="p">}</span>
  <span class="p">}</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">doAppend</span><span class="p">(</span><span class="n">ILoggingEvent</span> <span class="n">eventObject</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">LogbackException</span> <span class="p">{</span>
    <span class="k">try</span> <span class="p">{</span>
      <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
      <span class="n">OutputStream</span> <span class="n">locationOutputStream</span> <span class="o">=</span> <span class="n">locationManager</span><span class="p">.</span><span class="na">getLocationOutputStream</span> <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
      <span class="n">setOutputStream</span><span class="p">(</span><span class="n">locationOutputStream</span><span class="p">);</span>
      <span class="n">writeOut</span><span class="p">(</span><span class="n">eventObject</span><span class="p">);</span>
      <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
    <span class="p">}</span> <span class="k">catch</span>
     <span class="p">.</span> <span class="p">.</span> <span class="p">.</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Adding a dependency on the <code class="docutils literal notranslate"><span class="pre">cdap-watchdog</span></code> API will allow you to access the
<a class="reference external" href="https://github.com/cdapio/cdap/blob/develop/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/logging/AppenderContext.java">https://github.com/cdapio/cdap/blob/develop/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/logging/AppenderContext.java</a>
in your appender.</p>
</div>
</div>
<div class="section" id="enabling-access-log">
<span id="id1"></span><h2>Enabling Access Log<a class="headerlink" href="#enabling-access-log" title="Permalink to this headline">🔗</a></h2>
<p>Access logging can be enabled on distributed CDAP with security turned on. Once enabled, each HTTP access via the
Authentication Server and Router will be logged. Log output will be in the <a class="reference external" href="http://httpd.apache.org/docs/2.2/logs.html#accesslog">standard Apache HTTPd access log format</a>.</p>
<p>To enable the access logging, complete the following steps:</p>
<ul>
<li><p class="first">In the <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> file located in <code class="docutils literal notranslate"><span class="pre">/etc/cdap/conf</span></code>, have the following properties:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="o">&lt;</span><span class="n">appender</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;AUDIT&quot;</span> <span class="n">class</span><span class="o">=</span><span class="s">&quot;ch.qos.logback.core.rolling.RollingFileAppender&quot;</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">file</span><span class="o">&gt;</span><span class="n">access</span><span class="p">.</span><span class="na">log</span><span class="o">&lt;/</span><span class="n">file</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">rollingPolicy</span> <span class="n">class</span><span class="o">=</span><span class="s">&quot;ch.qos.logback.core.rolling.TimeBasedRollingPolicy&quot;</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">fileNamePattern</span><span class="o">&gt;</span><span class="n">access</span><span class="p">.</span><span class="na">log</span><span class="p">.</span><span class="o">%</span><span class="n">d</span><span class="p">{</span><span class="n">yyyy</span><span class="o">-</span><span class="n">MM</span><span class="o">-</span><span class="n">dd</span><span class="p">}</span><span class="o">&lt;/</span><span class="n">fileNamePattern</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">maxHistory</span><span class="o">&gt;</span><span class="mi">30</span><span class="o">&lt;/</span><span class="n">maxHistory</span><span class="o">&gt;</span>
  <span class="o">&lt;/</span><span class="n">rollingPolicy</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">encoder</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">pattern</span><span class="o">&gt;%</span><span class="n">msg</span><span class="o">%</span><span class="n">n</span><span class="o">&lt;/</span><span class="n">pattern</span><span class="o">&gt;</span>
  <span class="o">&lt;/</span><span class="n">encoder</span><span class="o">&gt;</span>
<span class="o">&lt;/</span><span class="n">appender</span><span class="o">&gt;</span>
<span class="o">&lt;</span><span class="n">logger</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;http-access&quot;</span> <span class="n">level</span><span class="o">=</span><span class="s">&quot;TRACE&quot;</span> <span class="n">additivity</span><span class="o">=</span><span class="s">&quot;false&quot;</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">appender</span><span class="o">-</span><span class="n">ref</span> <span class="n">ref</span><span class="o">=</span><span class="s">&quot;AUDIT&quot;</span> <span class="o">/&gt;</span>
<span class="o">&lt;/</span><span class="n">logger</span><span class="o">&gt;</span>

<span class="o">&lt;</span><span class="n">appender</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;EXTERNAL_AUTH_AUDIT&quot;</span> <span class="n">class</span><span class="o">=</span><span class="s">&quot;ch.qos.logback.core.rolling.RollingFileAppender&quot;</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">file</span><span class="o">&gt;</span><span class="n">external_auth_access</span><span class="p">.</span><span class="na">log</span><span class="o">&lt;/</span><span class="n">file</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">rollingPolicy</span> <span class="n">class</span><span class="o">=</span><span class="s">&quot;ch.qos.logback.core.rolling.TimeBasedRollingPolicy&quot;</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">fileNamePattern</span><span class="o">&gt;</span><span class="n">external_auth_access</span><span class="p">.</span><span class="na">log</span><span class="p">.</span><span class="o">%</span><span class="n">d</span><span class="p">{</span><span class="n">yyyy</span><span class="o">-</span><span class="n">MM</span><span class="o">-</span><span class="n">dd</span><span class="p">}</span><span class="o">&lt;/</span><span class="n">fileNamePattern</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">maxHistory</span><span class="o">&gt;</span><span class="mi">30</span><span class="o">&lt;/</span><span class="n">maxHistory</span><span class="o">&gt;</span>
  <span class="o">&lt;/</span><span class="n">rollingPolicy</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">encoder</span><span class="o">&gt;</span>
    <span class="o">&lt;</span><span class="n">pattern</span><span class="o">&gt;%</span><span class="n">msg</span><span class="o">%</span><span class="n">n</span><span class="o">&lt;/</span><span class="n">pattern</span><span class="o">&gt;</span>
  <span class="o">&lt;/</span><span class="n">encoder</span><span class="o">&gt;</span>
<span class="o">&lt;/</span><span class="n">appender</span><span class="o">&gt;</span>
<span class="o">&lt;</span><span class="n">logger</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;external-auth-access&quot;</span> <span class="n">level</span><span class="o">=</span><span class="s">&quot;TRACE&quot;</span> <span class="n">additivity</span><span class="o">=</span><span class="s">&quot;false&quot;</span><span class="o">&gt;</span>
  <span class="o">&lt;</span><span class="n">appender</span><span class="o">-</span><span class="n">ref</span> <span class="n">ref</span><span class="o">=</span><span class="s">&quot;EXTERNAL_AUTH_AUDIT&quot;</span> <span class="o">/&gt;</span>
<span class="o">&lt;/</span><span class="n">logger</span><span class="o">&gt;</span>
</pre></div>
</div>
<p><strong>Note:</strong> By default, these properties are already at the bottom of the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file but commented out.</p>
</li>
<li><p class="first">The log files <code class="docutils literal notranslate"><span class="pre">access.log</span></code> and <code class="docutils literal notranslate"><span class="pre">external_auth_access.log</span></code> will be available by default under <code class="docutils literal notranslate"><span class="pre">/home/cdap</span></code>
directory, to configure the log paths, simply provide the path of the log in the <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> file.
For example, having:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="o">&lt;</span><span class="n">file</span><span class="o">&gt;/</span><span class="n">var</span><span class="o">/</span><span class="n">log</span><span class="o">/</span><span class="n">cdap</span><span class="o">/</span><span class="n">access</span><span class="p">.</span><span class="na">log</span><span class="o">&lt;/</span><span class="n">file</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>will change the <code class="docutils literal notranslate"><span class="pre">access.log</span></code> file to the path <code class="docutils literal notranslate"><span class="pre">/var/log/cdap</span></code> directory.</p>
</li>
<li><p class="first">After modification of <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code>, restart <code class="docutils literal notranslate"><span class="pre">cdap-router</span></code> and <code class="docutils literal notranslate"><span class="pre">cdap-auth-server</span></code> using the following commands:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">$</span> <span class="o">/</span><span class="n">etc</span><span class="o">/</span><span class="n">init</span><span class="p">.</span><span class="na">d</span><span class="o">/</span><span class="n">cdap</span><span class="o">-</span><span class="n">router</span> <span class="n">restart</span>
<span class="n">$</span> <span class="o">/</span><span class="n">etc</span><span class="o">/</span><span class="n">init</span><span class="p">.</span><span class="na">d</span><span class="o">/</span><span class="n">cdap</span><span class="o">-</span><span class="n">auth</span><span class="o">-</span><span class="n">server</span> <span class="n">restart</span>
</pre></div>
</div>
</li>
</ul>
</div>
<div class="section" id="monitoring-utilities">
<h2>Monitoring Utilities<a class="headerlink" href="#monitoring-utilities" title="Permalink to this headline">🔗</a></h2>
<p>CDAP can be monitored using external systems such as <a class="reference external" href="https://www.nagios.org/">Nagios</a>;
a Nagios-style plugin <a class="reference external" href="https://github.com/cdapio/cdap-ops-tools/blob/develop/nagios/README.rst">is available</a> for
checking the status of CDAP applications, programs, and the CDAP instance itself.</p>
</div>
<div class="section" id="additional-references">
<h2>Additional References<a class="headerlink" href="#additional-references" title="Permalink to this headline">🔗</a></h2>
<p>For additional information beyond here, see the <span class="xref std std-ref">Logging</span>,
<span class="xref std std-ref">Metrics</span>, and <span class="xref std std-ref">Monitoring</span> HTTP RESTful APIs, the <span class="xref std std-ref">Java Client</span>, and the <a class="reference external" href="../../../developer-manual/advanced/application-logback.html#application-logback" title="(in Cask Data Application Platform v6.1.1)"><span>Application Logback</span></a>.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Operations" href="index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Metrics" href="metrics.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>